{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000,)\n",
      "(5000, 784)\n",
      "(5000,)\n",
      "(10000, 784)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('./')\n",
    "\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "\n",
    "print(mnist.validation.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "\n",
    "print(mnist.test.images.shape)\n",
    "print(mnist.test.labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 576x576 with 0 Axes>"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 576x576 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUMAAAD3CAYAAACHHzbQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO2dd3iUxdbAf7ObBiQECL2EEJJQRVFQ\npCiKWBBQ7BX1Ioig4rWX66de71Wv14YgKmAXRcULiqJYQQEVQaRD6L3XQCAku/P9cXaTLOlhS7I5\nv+fZZ9+dd953Tybvzpw5c+YcY61FURSlquMItQCKoigVAe0MFUVR0M5QURQF0M5QURQF0M5QURQF\n0M5QURQF0M5QURQFCHBnaIyxxpjDxph/l7L+IGPMIc91KYGULRzQ9g0c5WjbJz31rTEmItDyVWYq\n7HNrrQ3YC7BASr7PPYBDx70scHlx1+mr1O1bF5gN7AH2A78C3Uq6Tl8lt+1x527ynL/1uPIkT3lE\nqOWvyK/C2hZwAv8CtgIZwAKgVmn/J/54BXUEs9b+AsR6PxtjegJTgW+CKUcYcwj4G7AKeXAuAaYa\nY+pba3NCKlmYYIypDTwMLA21LGHGk0BX4ExgI9AOOBpMAUJtM7wJmGStPRxiOcICa+1Ra+1Ka60b\nMIALqA3UCa1kYcUzwCvA7lALEi54Bpi7gcHW2g1WWGKtrRqdoTGmOnAF8G6oZAhXjDGLkFH1C2C8\ntXZniEUKC4wxpwOdgNdDLUuYcRKQA1xhjNlujEk3xgwPthChNPRejoyuM0MoQ1hire1gjIkBBgBR\noZYnHDDGOIExwJ3WWrcxJtQihRNNgXggDWgBpAI/GGPSrbXfBUuIUE6TbwLesx7LqOJfPFPmj4CH\njDEnh1qeMGAYsMha+2uoBQlDjnje/2mtPWKtXQRMBPoEU4iQdIbGmGZAT+C9UHx/FSMSSA61EGFA\nL2CAZxq3HTH2v2CMGR1iucKBRZ73kCpGoZom3wjMsdauCdH3hyXGmC7I/3Qu4qpwF9AA+D2UcoUJ\nNwMx+T7/D5gEvBkSacIIa+0aY8wvwKPGmLuQwftq4NpgyhGqznAg8N8QfXc4E42sdCYD2cBi4GJr\n7daQShUGWGv35/9sjDkGHLTWHgiRSOHGtcjAsgfYCTxmrf0hqBIE2LnyKHAAeKqU9W9BnIWPAsmh\ndg6t6C9t3wrVto976h8FnKGWvyK/KupzazxfpiiKUqUJtdO1oihKhUA7Q0VRFEpYQOntuLJSz6G/\nc39aoT1jtX0Dh7Zt4AjXtlXNUFEUBe0MFUVRAO0MFUVRAO0MFUVRgNBGrVEURSkah5P0cR0BWHrB\nGAD63XQ7ABE/zPf/1/n9joqiKJUQ1QwVRalQRDRvBkD6Mwms6zneUyphOfe3lPe6Adi1rJqhoigK\nqhmGJc62aQCsuL02AKsuew23J1ScA/E3HbO/BQDvvtiHhDc1XqkSeiKSkwBY9mhdgHxaIQze1A2A\nBr9I6hlXIL4/APdUQkBEs6YALHu8IR+d+wYAHaPdALhx4MbtqSmTgSG1VgPQ+MEJvDW9BwA5m7cE\nUeKKjyNGwhcm/uwZQJrMxmmk/ZYfywTg3gsG4lq5OjQChhEmMorlT0jesnXn5XWCyd//DYBWQ5YB\n4D66KmAy6DRZURSFIGqG2+7pCoCxELNHpmz7Wsu5Rr+6iJk6N1iihBVrnzsTgBXXvwqAG5s7FXZ7\nxrqvMuOZe8g38v9pNdYDcHnsQbZOXwLAl+1qB0PkCo9XI9wyUUwJXzaZkHuu55JLATAvyFQues1f\nxd4rIikRgJz1G/0uZzixcvTJrDtvnE9ZyoybSR34J0DuvCaQqGaoKIpCOTTDncNFw9vfIRuAyeeX\nLh9Om6g/co+P2hwA4h3V5J43HmbrKyLKi9t7A7DnqpoA5GzaXFYRqxRX9p4NkLtAIrZBGeNe3d8S\ngO8uaFfAHji73zUA9H/9tVz74Zd0DobIFZ7VT4ij74rOr/qUp/5wK61uXwmA+/B6oPgMRuljO/P5\n+aMAuPqdewBIfGKOf4Wt5Kx+uYu89x2D97lN/k7shGlDlgY1Q1SpO8P0cfJDWdFnJADRJtJzJrrM\nX5p3rVDfWYP6Tjl+r/nPANzwcU8A9l2XqFOMwjj9JACGJrwGwFeZDQGYeyiZJQcbA5B1fz0A1jzn\nJO2p6gC4losB2muWiHzDSbbnidvyoAx0Tf5TdX+w9syT+fk6b3oeabONObJYkjZoCe7sYyXeI/u8\n0wCY3Hs07SI1bXVhHLtQ+pPJl74MgNPEkDLjZgBSb1kIgHUHYs24aHSarCiKQhk0w9fOkRTHXq3u\nP3tSAdh5LK7Q+v+bL6Nj4tSSY1Ru7uXguT4fAmLQB/ggaQYAN3zYk31Xi9uITpnzMXcxAEMul72a\nzm17Aa97zHYAtjwoiybLzx7FReMGS73lcvmeQbLwkm3n57rdNJ+wQe4ReOkrLDsePEZ9p2iER6xo\ngQPvvheA6tmly7h66O/yDJ8UFckhmwVAi0/3AIHxj6uMJDy6DoAOUbJY1Xt5P9Iel3ZzBVkj9KKa\noaIoCmXQDF+++goA/nGKLGzUnyKGZNeevYXWT+OPQssLI2UqjH+rDwDbJ8qCwPBamwDREFsNEe0n\n6THVDI/H/iEaYmHaXMxuMQaOPZBE1I5DAKx9UuyC79wohn0HhvlZMiaq0zUMSZuVezxg5ZUAVJ+c\npxGaCPnJmGrVClzrOkk08ZfavJ1b1nP+LQDUX7rC/8JWYu5q8r3P54PvNqXWqtDuhFLNUFEUhTJo\nhnb+UgASPGHE/D2rdy+SkfPtl/oCMPzJ13LPfXiDrGA/8tjpfv7W8OHIJdI2e1tH5GqECYtFGxwS\nv55TvhR74OnR+V1w4I8sB/8Y5LEn8mdQZa7oxEUeBeCw53P2+Z2o89h6AD5O/raQK2b6fJqd5aDe\ns2X3tghnDtwgrjRnxYizerdFlwFQ6/3fQiaTF92bHCZsvVqM/cvPfi3fDpS84AzeTtB7zjs1vnHS\nHST/pIEavIwd14+h94nv7HvJUwEYOudCAN5s/hoROEt9r5unDiX119D/yCsS+/sf9vmc+YW4hMXa\ntWW7kcMJfl5o0WmyoigKFUgz3PyIGPbdHTMKnGvgFK0n51xx14n40f8hv8OF/DtQ8keq8R4P2XQu\nAJseFtco1Qp9Odw0bxdsNSMO0+82/9FT4uTe7WKOmDZdnIazGx1j9fm+e2q91P2zwqY+DhmNah/0\n+VxtT+l2HWddJO29e7A4wLdvsI2MK+T/k7Ntu19kU81QURSFIGqG3sCNqwc1Ysw1Ywuc7xkjxntv\nvLj8NI2IBWDs27KQMqx59wBJWXlp/LGMklc26Uf7mlsBGJog2+qaOKvjHffWPNMGgGo/aZSgwkh7\nYxdtsocXei7l/b24V64BoEWOaNRrnz2zQL1hWyQQaZ0P5wd1b21FJ6JhA8a18kYAii2xvrNWPJf+\nKttHr457BciLZwDQbvT1ADS93D+aYcA6w0NXngHArlPlR/jPyyYCcE3cviKuKFlJPe/7uwFIY96J\nCxhmVPtcOresz2G+N4BrZ/HPzHjqMD+e9DEA3Z8Qg/7C+ZJnQn0LfXGlr6HFQ2sKP1dIWURmwanw\nvPGnAFA3W00QPkRGkhhRcie4c5iYzC69bQZD4rd6Sgv6ddaLO1yg7ETQabKiKAp+1gxNx3YA1Bq9\njWlJ4idY2LR3ymEZHZYcaZpb9uVzPaV+lkwsbvqnuDXkjQwQtd032k1VwRvSv6x7s727U2IvhCtn\n9gNgcso0ANrfKqaGxCdUMzwRTD51McejO9ZOzwqRNBUbm5HB2AMSUSn/7xrAWTeBTX9rBcDiu8eU\n6n4Hjsi+5vp+kk81Q0VRFPykGW7w7Hd97BqxS10ftyc3BtyKYxJK/s6PbgWg+jZDoxmeDFfL0nPv\nEY+vc+qqhxt4TmxlXbbspEj6/JA/xK1UHLnk9Fw735cbRPNudOnyMt/nwPMSft79umje2alH/CRh\n1eaWa6fnHl+5WrRv5wzdyVMYrv0H+GizuMgMif8cgG4Pyr7vzk+t5arY0idDfnJXWxrfJTZDf0VZ\nUs1QURQFP2mGtTrvBEQjBOi1rD/Zo2SbjXeVM4m8lbXiNtG4z5aQ65fWetNT4mCv2xMt2BPDryrg\ntRNe/czXzDuYBJRPIwRxUbjiWdFgvNvxlBPDWU+iiKdG56UJ3f1aEgBx+MfVIxw5+nYjALL+K2lD\n/ttwQamuy7bSa7SdOQiAtIf3kLNhk19l80tnmDBIpq8p94grR8v7fyWC8oXq35cmRtFuMXlK65Al\nNwBQl/RCrwlHNlwn09oh8Z/z0oLzAGhJ6R6cXDypAS56++fcPCfejHmR6QVdFZTSc+AcyS/Tr/r0\n3ACuMbuzQylSpaDmh2Ly+f1fshh6VkzRdV3WTad51wEQNUnMbcnvi1IViADEOk1WFEXBT5qhd29g\ny/tPfHqwp7Nvn7/8WCZxY+JP+L6VjSY/yR7tyBFORpwie2PfvPNiABKWZhXYn+1smwbA1l51ib1Y\n/g8/nfQOIFNjr0aY9vVt8v5k1U365A9uevKL3ON12R5t+3vdM19eWs+6EbNEUoi0eEXCBVqXm/oZ\nwQuKq5qhoigKFShqzQVLJJrF5FreXLWyaHLT0puo/XXpUwiEDZ7Fom6LLsvdSjf0IQnV78bNkztP\n86neP/4jADpGu3EUErWm1STZb9v2v2J0rspJn/xBgjPPzev5bRd4jvaHRphKTNvXhgGQ9MxcbI48\nlaFKmlVhOsMrai4CoLpDdqekZ4sPUfXRtUImU0Wg1uBjPPmFdHxPN5A2yrbwVH2JFJw/gKt8drDD\nJT6EY/aI/+e3o7uR+mbgDM9VnWPu0gd8VYR/J8v+7WaIuaYiBLTQabKiKAoVRDPcOawrDZwyFfbu\nNrn26fsBqPt11Y78kbNpMwv7SYSZlP/kTY2X9xwPwFmLrgJg196auedSXhb9z7s3OYGq3YaBZlzS\nlwCc9sLfAWh5r4b6r4yoZqgoikKANcPv7SQcOEkklRTTvsB5Ey2Zwy4f+iMZ7mNM+CiTex87gj16\njK58R3VTcuyzqoA35mDL6/MizPTlNE/7Pl+gfYuyv+yxO1jEHFy46EgPEkyDQIpdqSnp2X10ogQW\nbT3wRVpHRvP2hwdZ/8gj2GPHaMSF+uwWQ0ltezxr7FI2kI4bF+dyGY5CImH5g4BPk7vQ2+fBWG7n\ns49dZHKIdjldaBzRMvfc9ddW5/UW95B+xeOBFitsOL5999qdrGIRmRwiimia04qmRpKbJ5gGnMMA\nZtlpoRK3UlHcs9vgTyc1T81LXXvLdTV5lodZd9+9oRC10nF822bY/SxjHofJoAZxtKUTcUYWT1ua\ndjS2Sczm64DKFHSbYSzxNKApq1giKozb8v7Uc/h6YU8AEj/5rQptuvMvbutmIXNIpQNNaMFB9vEn\nM4m3dXIfLKX85H92EyavofEUF/f8X17Y/5b8yroQyldZ8T63zUihGS3ZzFoWMoeu9sKAaYGFEfTO\nsJlJAcBhl2FdOVj3MZIeVQO/P8jmGC5yaEQixhjiqUMNW5PDHCQO7QxPlPzPruI/9rETi5tEUjHG\nkEgqG2w6e9lJXRoGTQ5dQAkjok0MDWjGVtZjrWW/3cMRMqlF3VCLpihFcoiDxBKPMXkRleKI5zAH\ni7kqAFhrA/ZCJsIpRZybBdxc1uv0VXw7Af2AHYh/dQ4wuJDr1gPnhVr+ivzSZzd4bQs8Bkw8rs4E\n4Il8n5M810UESi7VDMMIY0xr4GNgILKfsR3wgDHm4pAKpijFcwioeVxZTSAjmEJoZxhetAdWWmun\nW2vd1tqVwFfARSGWS1GKYynQweSfJ0MHT3nQCHpnaIyJMsbEAAaINMbEGBPEJaPwZgGQaow51wgt\ngb7AwhDLFRbosxswZiDxGe4yxkQbY+7wlP8YTCFC8Y/8FjgCdAXGeo7PCoEcYYe1dg3wN+AV4CAw\nE/gMeLO465RSo89uALDWHgMuRcw7+5Fn+FJPeVAFCaSh9ChwAHiqlPVv8TTGUSA51Ibeiv4qR/v2\n8rTvEeCcUMtfkV/67Faotn3cU/8o4AyUXMbzZYqiKFUatXcoiqKgnaGiKApQwna83o4rK/Uc+jv3\npxU6SbC2b+DQtg0c4dq2qhkqiqKgnaGiKAqgnaGiKAqgnaGiKAqgnaGiKAoQ4ux46yZ2AGBWt9e4\nbuCdADh/+jOUIilKmVjzQhdGXCjh6KddK1Gv3YtWhFKkyksX6Q/WjZDF3vSz3yVlxs0AtLzur4B/\nvWqGiqIohFgztBtrAJDQoxp7W0mmvHo/hVKi8CTr4s7sHSz5qBd0nlDg/NDNPQCY9fXJACS/sZac\nbduDJ2AlJKJJYwBGX/I2vasdAeDdM/oAkLAoZGJVWrbf3ZWn73gLgPOrHQYg28LI0ycC8Aqtferv\nuLMrjT8UDdy1Z69fZAhpZ1hjc57vY8OrNwDgej1U0oQPJjIKgPQXOwLwVb+XSImUwcZdSP3Xm/4i\n5wb/DMApJw2k6eXaGRbHmtuaA+R2hErZ8KYJ3nfVqQD8fN8LVDdRJV63+eGuAPwx/GU+Gd4UgFde\nvhyAeq+fWC4lnSYriqIQYs0wP0dyIgGJVa+cGCtHnQJAer8xADiIwV1Eavkhm3oyvtlMn7JXTpnI\nCwlnA/6bgoQbzbptDrUIlZq1T4hGuHTgaE9JwV/+6/uTeeN9yVjRhDkAZCXI3CbSOLk+bhsAnR96\nEYAbueeEtEPVDBVFUQixZljz4m25xwc+E4N0PTaESpxKjddOuHLUKSzt6x1tnQBsc2Vy1uT7AEie\nLMGDo1eJTdC1ew8dP74egPmdPwDgzyNJ2GPZwRK9UnG07+kAjEwe5SmJDJ0wlRCvrbBG231F1vk6\nMw6Azx44nyZfzSnxnmmeZ3/iw89zQce7pey2P8osW0g6Q1dPUZGntnsVgL+OOWkwYQlQuIFfKZlt\nwzsBkN5vFN5O8M0DiQD8b3BvUmf/5lM/J99xVpbvD3rqlg5Uy1gXMFkrM0cSpG1PitJOsKyYiAjW\n/FN++8s6jS5wfsimngDsvFw6w+gtBTu0pK9kMO/Q/GbmnynZLCKN/E9aRMRQc0X5/y86TVYURSFU\nmmG09MGxRlTmbGtxZwQ1RWrYcfuQzwFwYHhmT1sAfu2fBoBZX9B731lT0tRuvrU9D3T4HwALjole\nXu0C1QrLwuwseZ7jNuWUULNqk3VeR5bdUFAjBBixtRs7LhatzrVna5H38O5QS/wJJq9sBMBVsTv9\nIp9qhoqiKIRIM1w/QPtgf+PyjGtuLNOe7glA3Pp8dkKH2FVcZ8suk76jfwBgaK2fcCDO7xevvNRT\neUvgBa6ktB5aMK/5y5t7AxD1TdmN9lWBHXeJo/Sw26cUODdiazcA1p3twJ0ZWjcu7ZUURVEIkWYY\n11Dtg4Gk+vaCube9GuHXH4wrcG7AatlT67g8U+oGULbKzrAG3s3zeVtJV36dCkBTdoVAooqL4+Q2\nADx7l6z69qqWmXvOu3LstROWVSs0HduRFOkb4Wp1dhbxa8tvt60wO1CUE2PVkQZyEL+et957BYBn\nd5wHwIwNKXxz+iuemtUAOOA+CkDnr/5O63tl6uc+fDh4AocRzadIJ6iDiC893pfOKn8n6OWPKScB\n0GRPyX6EhbHy9uqcHu27q2r64bZU+3xuue4HOk1WFEUBQqAZOmJi6N7E13Vj3M6zgUPBFiWsWD5c\n3Gn47HcaOUX7G9l4NgCOxnNwezRCL+eMuh+AtOfmqKN7KdlxV1daRXo1mRgAtrgyIUd1wuPZfduZ\n3F77Bc8ncaHb5jrCPRtkkS7xfzuAsmvTES0kWtDMC1+C457pWXtTgN3llFg1Q0VRFCAUmmGteEY1\n/tqnbOas9rTktyKuUIoj6+LOAGy6RgzHDgrmx3YaB1jR/3otvQyAxs+Vz1ZTFXE2qA9Ax+sWU9MR\n43Ou5+T7SF2lz+7xZDSHWEe0T9nzO88ho4dXcyufBrdyuDhae2c/APs89u/tI1tS4wQ0w6B3hjlJ\nDQqUJX6jQQHKgqNDaxqOFV/A8c3eAMgN0ZV/yvvQduko/ze3E6/1fheAN1tJMIaBV0nghthP9Idc\nInVrAzC+2Te5RQc9P8C4dTq5Ki3ffN+JFpQzxJaRQd46C566b/NFANSY9Ht5RQN0mqwoigKEQDPc\n/ejR3OM+K/oDEDVjYRGhR5X87B4i2demP/Y88bnTNd9p8b3buvD1jxLBJu0lWahK2zaX58+RMF1e\nP8NrHhdTxZef1A602JUeV42CgUcXZ1cHoOHLam4oLY1ml3+h6cD1ZwCw4qpXC5ybM1sWD0/U1Kaa\noaIoCiHQDF9rPwFvvL2tByVySuMcDaFeHBnXdAFEIwSId8SwPFvsrC9tl32xK19uJ+em/EXyUbHL\n5PfFd85cCEDrT4YDsPDKlwGYfP4dRH47L7B/QCUn7oVtBcpuXyCadlMK7lVWCqf5IyvYMbX09SOa\nNmHVcInJ+fsNvm46AB9lyPpD2tsSKPZEHZxUM1QURSGImmFEkvTwcWYOTqNRgsvC7g5iF/TaCScf\nrsPbV0miHPdfywCI89hLinKgdlSTa9uduh6AaM//wB1R0BVHESKaSSrKtNiNuWXXr5ctjs1vlZh7\n6m5derrXWs2UVJnluFatLXDe2Ub2eK+6qS4AL1/xdm4O5fwaoZd3h18CQMTS+X6RL2id4dHx8p4W\nGYPL4/MW+0nNYH19WOD1IXzwp6tI+6v04aKcdROoPlmu/Th5mqdUO8GS2N6nGQBf1P8CEH/NfUdl\n4cRxTKZmJjIKm10wMEZVJ3X8Np7sI1kaH68nwYVvqbkJ5xfy21+c2bTANafUkCyN3qx3hfHFYVnw\nu+/7a2j9m2dPvZ9k1mmyoigKQdAMnWktAbg36YvcsmvXidG/5sQTc5KsKtRdJI5H+9xHAPijz8t0\nfkOygLX5P8km6NqRF/o8oolkGjx8chMA7h75ERdXPwDkjaKv7pf/S7VfVuje5FLism6mtfY8x+ny\nljppGKkj1HH9eHLWrmf6K90BuPtJaZ94RwwDa3oCB9csXQDhTCta96t7Rcv8+W+ykSBt3ly/P7eq\nGSqKohAEzfBYk3gAelXLyi1L/7gVAA2sOqyWhriJMrKelSKRZhbePor0vq8DsPR8caC5e9XVufUn\ntJkA5C24ODC5o+i928SAveJOcVQ1GQsDK3wlJmavtNqaHNHIW0bk7Yc94tFYqm9TfaIo6rwlLl7/\nd3svAIbWm0GbyNIvnr66vyXvj5StdnXHerfxLfGrjPkJup/h0M09aPzRSkBX4spKnRXSYq/vT6Zt\njPhm9oyRhZDv2n2Wr6ZvMIHXDzTnpa/6ApD62AIAzFHtBEsi9lMx41zVUAahvx4ew792twbgs7Hn\nAtBktA7oJbGms+w6eyjlWtbc3BCACy4U39YXGslA3+69OzDHdQgtP9xD3WXl3MtcDnRYUxRFAYy1\nRe8K7u24slJvGf7O/WmF9h85kfb1+m2ueraWT/kzp05hTkYKAFOny37OFo8EZnStyO2rz27gCNe2\nVc1QURQFTQhVaclZL7siWlyz0ad8LMl4HWjKHTtOUaogAe0Mv7eTcOAkkVRSTPsS62+x60jnL1y4\n6MqFVDexgRSv0lPW9t1jd7CIObhw0ZEeJJiCgXYVoaxtu8YuZQPpuHFxLpfhMDrpKoqK2i8EXDPs\nQm8f4XfZraxmCUc5TCzxtKETsUa25TUxLWhCC763kwItVthwfPvutTtZxSIyOUQU0TSnFU1NMgAJ\npgHnMIBZdlpRt1PycXzbZtj9LGMeh8mgBnG0pRNxRmy2LU07GtskZvN1UbdT8pG/bY/ZLBYyh0wy\nsFhqEEcqHahlZI9ysPqFoE6TM20GS5hLR7pTkzpsIJ2FzOZMe4GOpH7Abd0sZA6pdKAJLTjIPv5k\nJvG2Tu6PVikf3rZtRgrNaMlm1rKQOXS1F+qze4I4iaAtnaiOdI672MpfzOYs2y+obRvU/+IedlCL\nutQydXEYB0m0Iosj7GdXMMUIW7I5hoscGpGIMYZ4U4ca1OQwB0MtWqVnHzuxuEkkFYdxkmhSsVj2\nsrPki5VicRonNUwcxpPnxGDIIZscghwAw1obsBdggZR8n+8EpuX77ASOAiOKu05fpWtfT9mHwHBP\n254J7ASaHVdnPXBeqOWvyK9Cnt2/A18fV+dL4N58n5M810WEWv6K/Crq9w0sAo55zo8r7XX+egVb\nv/8OONsY09MYEwU8AkQB1YMsRzjzEfB/QBbwC/CotXZTaEUKC2KBA8eVHQDiQiBLWGKt7QDUBK4D\nZgX7+4PaGVprVwA3AaOBbUBdYBmgcf/9gDGmNfAxMBAZZNoBDxhjLg6pYOHBIeSHmp+aQEYIZAlb\nrLVHrbUfAQ8ZY04O5ncH3fJrrZ1krW1vrU0AHgeaA6WPVKoUR3tgpbV2urXWba1dCXwFXBRiucKB\npUAH4zVsCR085Yr/iQSSg/mFQe8MjTGnGWOcxph6wBvAVI/GqJw4C4BUY8y5RmgJ9AU0KsOJMwOJ\nLXKXMSbaGHOHp/zH0IkUHhhjuhhjuhtjoowx1YwxDwINgKAGPA2FT8BIYD+w0vM+OAQyhCXW2jXA\n34BXgIPATOAz4M1QyhUOWGuPAZciJoj9SDtf6ilXToxo4FVgD7AF6ANcbK3dGlQpArxqdBQxMj9V\nyvq3IA/aUSA51KteFf1Vjvbt5WnfI8A5oZa/Ir/K0baPe+ofBZyhlr8ivypqv1Bs1BpFUZSqgrrO\nK4qioJ2hoigKUMLe5HAN4lhR0PYNHNq2gSNc21Y1Q0VRFLQzVBRFAbQzVBRFAbQzVBRFAbQzVBRF\nATQhVNgR0bwZ+89oAsC2vrJT7PZTZ3J37XQA2s+6BQD3+hoApDy5EHdmpu89GjUkZ9v2YImsVEFy\nep0GwJ520QAcqW+xKYcBePDkbwEYFL+dbzLl/P1jBwHQ+Lk5AZNJNUNFURRUMwwbtt7fFYBHb/2I\nAbG+oegdOHB70ocu6u6J2dBd3k4+OoLmj/uOttEfu8g5K7DyVgo80bp23n4mALffOYUh8UXHDhh7\noDEAU/p3AcC9fjM2W+M45BqRrSMAABj/SURBVOfADdI2Pz77CgDRRrogN3muiw6k3bOtpVc1mbXM\nuusFALo67wWg6TP+1xCD3hk6Tm7DynuqAXDjKRKh5846c+n1wv0ANHw5cGpwOOJsmwZIJwj4dIS7\nXFkAbMipjotIADpFyY/T6fmhL7x1JJ0PjgCg0QvS9t3rrGF6gTimVQiHE4BNj54BwOKho3NPZVkX\nAFtzpG1jDNR3SqD2QTUlRvGgGZLFbeS+FH7oK6kwvXmuqzoHLz0EQKSRNvZ2ghtzjvDo5v4+dX9f\nkUxkDXleZ3V7DYCul0o0uk0vRmOzsvwqm06TFUVRCIJmaKLFALp9iBhMf39oJBlu6e27TLwPgJ9P\nSeHsGyTY9cqXAy1ReLHiIUmv6NUIM9zHOGeehIhsMDIGAOeMP3Pr775Npnx9h/0MwCN1/8IV7XvP\nWXtbQhXOWLjl/oIaIUCWzeHkD0WLTn7gVwCcbVJZ8bCkQVly7utA3tRvRO3VkjIK+L5nCwBcu/cE\nVvgKTtLgLQAM+0bsMEv2NgSg9ghwpa/xqZvG3tzjM17/OwDp/URDPOXeO2n6tH9nkaoZKoqiEEDN\n0BEjWsmKlzsAsLqfjLKj9qfy6ZMXAtDyE8/omtaSRS1PAcD2E1tWRKbYZiJ+mB8oEcOC//V4zXMk\n49qwDf1pPGBZkfXrviFt/uNOWUF5ZPRfBeqs/CaVplVUMzQREUR1K1x7a/+/O0n1aIReXMtXkTpQ\njnsMEa3xuQfHAtAzJlu0Q+CHuJOkUhXXDF379gGwYJzMUGqtEbufK/3PIq8BcB721dva9VnJgaf9\nK1tAOkNH9eps+bA5AKs7y9ThxX2pAEy/82xif/rNp74rfQ3V90mi87t/nQHA+O2iRh/4IRAShg8n\nRcnCiNcQ/Ud6C9Io+QcXt0Q6u1lHY0hYmuNzzlbYeCmBx5nYlD9O+8inbNR+yUvU+vV9uIq5tu5Y\n6SgnD+4EQM/GvxZTu2qTMP7E2qZv3YVMoKmfpBF0mqwoioKfNUNHdXExWPFC+1yN8Pm9rQD4uX9b\nAJzrCleHN90smmOvatMB2FtP6r1XqwOu/cfn7la8nLPkcgC+a/8JAO/2HM+/OaXI+l7P/3pPibE6\nOeIAde9dB8Dhz6WOqdTR6k6M9Vc3zj0+ZGUKN/FpMevEL/ut0GuOZ+3NSQDMnvo73aLFv3PVELlv\n8mNbsDk5RV2qHEfWRZ0BuLn3DJ/yKTs74u9FPtUMFUVR8LNmuOv6kwFY3f9VvsoUl4+fL2kHQM66\n9cVeeyzeVx1ZflRGUtUKiyf2bvkXvjZJNOsh8emkjzkdgLb/2QbAjvOb0u+OmQAMrDUSgMYRXn+a\naN5LngpA3z53ApBTreqphs6EOgA8eNMnuWWTMsQdJn5C6TRCL66lKwG4afoQVveXGdLyga8CcPFn\nA2HekhOWN5xx1hSH/x3XtuO2u2W64nVoX59zBIA9/21BjGqGiqIo/scvmmFEE9HiHrj/QwC2uDJ5\n5vFhANRcW/KoGpGcRN+LfveHKFUO1/JVALw/8iIAbn98FSsuES2ES+Qt/95kydedx4Pbz2Tqz7L6\n2XqxjL63PbeM6Y9Vre14xuMKdn3czhJqlp6aKyLAd4cZK4fGkHar376i0uM4RdYStvasxcFWYksd\n3E1mMfcn/JSvprg4nDftHgDSps71uyx+6QzdCfLDubyG+BD9c/cZ1Pyw6E7QRMjXbrlbpnMPDf6Y\na2Krpl/biXLkEmnDHrf9Uar6gzb0BmDXPYkAOBatJiVT/ldq1vflp32tPUf7QypHuBDRqCE3zRSX\nmguqS4i4SKRT8+5VLoru94lylfZx6Z7z8qDTZEVRFALkdN2/5gK+9HjjR2b6GuP3XnyEL7uOAaBl\nhIwKUw7XIuWLoQC5Buc/9jb3XFF0yKSqzN5bxIP/qnslEKY3eGth41ukcdL2VVkcafZv735O0Xbc\nBWqDwxRWGt6svTWpQNmSiTKFa4BGUvIHtnZNBtTw7jeOKtO1ue5e7uLc3k8M1QwVRVHwl81wsbgS\npH0i8/r0q8Yw9/FXi6z/zZEEAC4d/zcAEp+bT+tWsh3Pa3Be9YdohsmqGRYgonkzHnvkXQAuqp4B\nkLtAsteVRf9F0q7vtX8HgJTIaCKOlv7+blv1xsijzTUIa8DZtosz5l8HQMf6Er3mlx9lz3a1HXl7\nQI80EDXwn5dP5PLY3QD0eWQGANPoCUDcxLK5O5UG/0yTrQif8ncR8PQVw3H32edTZf9OCXOU9BlE\nfSNG0Gae6YcF7KIVAPxrtwTDvOECWVGa80DZ1OlwxtkqBYBnpn9Aq0gxOG/0BBnt84EEx00Zs4E6\nW2TK3Pf94QCsOHc8dS7wDCoveQzVxUw33vzwQprq1FDxM659+6jXX/qFzZ6yFhS9R/n9UWcw6m3Z\n1fbjSR8DMHOw+NPyidPvU+aqpwIoiqIUQkAWUOq+8Su84VtWv4RrvDsAOlYX7XJ+ZosASFa5WfW4\n7OppFenk+yOiaT/x77sASHpbRtj87jEpNy4A4PKZFzO93acAdBkmC1v1Rxet+fk7aGZlY5tL8m7U\n3Oh/Z6Maq3WmU1pytm0nVraFc+8fEnJuWuspAHQZfEduODp/oZqhoigKFSg7nm0iuuPF1SVhzIhf\nxHUkjXkhk6mi8U6Xt3KP/zviRgDqfFXy6Ljmm2S4Q45vHSb7kL8YneB/AcOEOE9CqKya8l6tjNc7\n24hd64bB0wuca/7u2irj3O6sXRsAe0wWp9yHD5f7Xt/83BGAl66RWcuA4T/xyxsxJyihL6oZKoqi\nUIE0wy296/h8jtgdGSJJKi5OTzRrBw6i95Q+TWLSO2v5YGAzALpVkzD0X9WVFKNVPUGRl7ilHlve\nBRBrZP/2mSPE62H5e2W7V5N3ZOX+ntqrcsvavCsr+8m7AredrCIR0awpbT8X95kvP5dZXuKTZbNF\nm+hoNt4v8Tcf6DPF51xi1G7wc6TrCtMZZtWuemGjysoHeyRRfMfGs1gvycJIfkZ2Sbj/Kjrvic1x\nccAlLgptomQysHOAdIYJ4wpOszOu6RIQP66KTLOJ6+Xgnryyk6qLA8hyGpbqHmuflR/9J01e9JRE\nM+6ADEIpL8kg5KoigV0PnN6EZxt8AcAjt84G4LS6f6fV+INFXrP2yloAZNcWn9mnzpvEVbHSgXoT\ny3v3Ro156gri8e8zqtNkRVEUKpBmqJTMt9+fKgcDZ7Go+5sAbP1cpssv7OwFwNe/dCxw3eTLXs51\n0l6QJeNfvQkLgcL3Jl/xj2+ZPrFqhfCyHuP+yH0puRntro3bCMC/3+sDQKvnM3F7Ngfk59CVkmd5\nwQ0vAVDNM80ed6AZX1wu2rxr16oC14UzNbYcyd1A8Y+6Esx25WVjcFzm1fC8Jh/fz0WV7fS4O3X7\n/F4A0iYtwN9zSdUMFUVRqICaodNI/1x7aYgFqYCkvCxJnH6/OpIzorMBaBohjh8vNJ4l71fPKnCd\ng8jcvctfZ0gea3dmZpHfM255NxJZ7D/BKwHe9BI/9G0PX0qZV0Nc1Ws8AO+f3pD/TLzC57rrL/uR\n6+NfAKCaqe5zbtQHl9B0eRV1YP9tET/fIzbU8x+WbaT/a/1x7uKUV/vz4sDkaoITMsTN7orYrbT7\nRuIdNJ8s9VO/kiDQgVhhqHCdocvKj7b28kMhlqTi4dohUZifvfByVg6rB8CQXpJY+u46RS+gDNp4\nDn9MlylL8psbPaWbi6yfeGXV6gjzk7N+Ix+OvEA+jPC8eTrFG+O2c+Pg0YVcJZ3gOwcl4vtnV5wN\nQNPlVTt6e8QP8+XAk/u8f78RbL1WfA7n9pAwflesvAaA3V82xRs5rvEEMUW8e/IlpP0YPD9jnSYr\niqJQATVD7zRZKRpX+hpS7pYp84/U8Lx3LuaKgyR6otBUDceOE8PrbvTtO3UB+D5J8lCvuKM+3U8X\nDXzW3La59VuPlUgs7nTJP22zVwZN1spEzNS5eBIxcg2ysBSBzFQasjG3njcWTcSPewkm2vMoiqIQ\nYM3wezsJB04SSSXFtC+x/sF5vxP37zVkZlq6t9hCjajaBC7Id+WnrO27xa4jnb9w4aIrF1LdxAZB\nysrJ93YSjmOetl0ldq7UEWvZ4Tmfms/h14W2bVko63O7xi5lA+m4cXEul+EI0Owx4NPkLvT2eTCW\n2/nsYxeZHKItnWhsknLP1ex0BrNG/kxa4vZAixU25G/fwzaDVSziAHuwWGpSh1acQg0j4b6amBY0\noQXf20mhFLnSkL9t99ld/IXvSr0LFyfRhQamqbZtGTm+X8iw+1nGPA6TQQ3iaEsn4ozsSGlp2tHY\nJjGbrwMqU9BthrHE04CmrGKJT3nSP8ROM+wf3YFJuNdtxGWCazOo7OSQTT0a047OOIlgHctYyBy6\nckGoRav01Db1OIcBuZ/32p0sZA51S7lVTykat3WzkDk0I4VmtGQza+W5tRcGTAssjKDbDJuZFOqY\nBjjUXOl34k0dmpgWRJooHMZBImlkksExW/qgDkrp2MYG6tMEp6lwa5CVjn3sxOImkVQcxkmiScVi\n2cvOoMqhPVIYs49dRBFDlMfRVfEPLpvDTrbQiOYlV1ZK5BAHiSUeY/IcseOI5zBFB3UIBMbawEWL\nMcZYINVau7qQc7OA8dbad8pynZJHCe3bFPgduM9a+1Fpr1OEEtr2RuCfQLI97gekbVsyx7eRMeYx\noJ219pp8dSYAq6y1T3g+JwHrgEhrbUA8xFQzDEOMMfWAb4Exx3eEil+4CXjv+I5QKTeHgOMjg9QE\nMoIphHaGYYYxpjbSEX5hrf13qOUJN4wxzYCeQBlDvirFsBToYPLPk6GDpzxoBL0zNMZEGWNiAANE\nGmNijNFtJ/7AGFMTmA7MttY+FGp5wpQbgTnW2jWhFiSMmIG4a95ljIk2xngy9vBjMIUIRSf0LXAE\n6AqM9RyfFQI5wpEBQGfgFmPMoXyvxFALFkYMBN4NtRDhhLX2GHAp0rb7gb8Bl3rKg0agO8MsYL4x\n5ilvgbW2p7XWHPeaAWCMucUYs99zXWFxRxVffNrXWvuupz1rWGtj8702grZvGSnw7AJYa1tba988\nvrK2bZkorF9YYK09zVpbzVp7qrV2gfecMeZxYKHnuoDZaQO6mqwoilJZUFudoigK2hkqiqIA2hkq\niqIAJQRq6O24slIbFL9zf2pKrhU6tH0Dh7Zt4AjXtlXNUFEUBe0MFUVRAO0MFUVRAO0MFUVRAO0M\nFUVRgBClCl317qkArDxvHADn3jGM6pOrdsJtRVEEZ7tWAKy/LIFOfSQ9yHvNfwYg2xZMEddr+O0A\nVJsy94S+VzVDRVEUQpVE3oqbj9uzn31LL0idHBJJKi0RLSTk/KYBTQDISJPgv63StjC11RcApH05\nFICm0x3UXCAZB+2hTABcu3YBYCIi2HrX6QDkVJN7Jz4/H5uleVOU4HLwui4AXPzQDAAmJyzOPZdt\nRW9zFxID47WXRwJw/8qBuJavKvf3V4hsNi3bbMVES54O/RGWzPa7uzLv/lFA4Q+HtyS97+vyuW9e\nnY8zGgHw1t8l09vWHhEsvmmkz/X9ZgzGzP7L32IrSgEcMTEArHmiI0tvHA0U/kwXR1pkFADLR9Qm\nbegJyFL+SxVFUcKHCqEZTms9hUtiewPgUs2wSJwpLQB4d8RLFPevm3yoPgCXx+4ucO7quG3yPn4M\nAA4cuePwgiwZG50Hjla5gHw77uoKwMFOR8t9j8hoMVUs6f52blnfJqedmGDhiifC/5onOgKw+MZX\nKI1u1vaTO3OPl101yufcM+d8ytun95UPcxdTVlQzVBRFoYJohkrp2NpH7H1tovLGsHMXXw1Ajafy\nkotFbtsPwJuNawGQlRDFsOc+BWBAbMHE3EuOyb77++8dBkD1JVXPzelwF1lYWn72uNwyB0Ub7fPX\nOf6899MHB5v5V8gwwN1DNMG1Q+TzsnNfKVBn0qGGAPxj1gCafSH/g2qfi9tMCr8BYDq2g6t8rxsQ\nu5NXkmsAEFcOLxvtDCsR3W+cn3u8zXUEgB2LGwDgvCivXoN5YlDe0ckp1523uNBO0MuXB08BqNK+\nnqnD1gFwWdwA1t0sKWOyaku3ZoqJ0eKue4zl573hU9Z6mgwqbR5YDezzv7CVFWPydYJjC5zut7I/\nAO7H6gGQNnte0EQDnSYriqIAqhlWKr6adzIAz/X7hcSIWACWXze6YMVb5C3SiGYoXvsy7u32aJQ9\nPr0PgBlXPs8jdcXY3POq4QDEfvJbQOSvyLj2H5CD/Qdo9tTmUl936KoucJ4cr86WBZQ2/90r99yn\nWiH4us8UNi0G+D0rEnvuFgAMW4ImW35UM1QURUE1w0pF2u1iFT41YRCLu70DFG/cz/bYur44XJuR\n63oB4BhZF4CW00T761HjHlb0exWArb1l32faJ34XPWzZ1jcvte+Tm8Wtw5Wu+eXzY9u0BLzuM760\n+eE2AFqOdeMgtI7+qhkqiqKgmmGlJPnJY/RsN7zU9WvN2061tes8n9YVWe+ktE2AZOpWSseqXuNx\ne3SK+XNTAUhhTyhFqnBs6RUP5LkqAUw+XAeA1NHZUlAuJ2m5X55tPDfsQbnQzrAS4lq6ktilpa+f\nU8y5Vml5xurF6eIXl8b2ckpW9XBjc00VxbngVEUimjUF4MLrfgV8TToP/ij+sWlzyxd2a/Njeffz\nmoNuWn8etb9aBkDBQF8lo9NkRVEUQqUZeobQ49VcJXhknyd7Zqe3GsuvWZEAtBojuzBUwSmZI5ec\n7jnKc4R31ZEp39oPxYn9tOYbubvRd3IOmb8NfusOmv1rTvAEDSF7e4hm+K8GefH5ei+RbSNtHlgB\nlF2DW/9xBwDeOuWdAufWvN6aWgd/LbugHlQzVBRFoYIEd822sPxpWX5Pu21vSESqKjji4gB4eqxs\nh4o0Tn4+1BoAu6AMhsgwxdmgPhldJTrQkTqiKzguKxj95912L3uOonPLVpz/eoF6gzZINKb537QF\nIOnFv6pMRKA9/TMLlG3anABA2sGiF/KK44EO3wLQKTpPpxy08RwAEr5ZXS5boZeKs4ASVVUekdDg\nTJDVu0Mfyspex+i8geitmWcDkErV3ZucfX4nAOIeW8/kZNnVU3yghsgCJd6Ob9c9iXmFvy0CIJE5\nnntVHR455RvAdxU5bVD59hsf/FqUpYE1vWaJvHsue6sdAAm7yj9F9r2joihKFabiaIZKQNk0SKbC\n89r7hvj/1+4OtHlpB1C8C064s+Ei+SlMT57OhAzJK7PfVR2Az7fKnvCdPzXJrf/KIIlU06uai85/\nXgtAnb7pnrP7gyFyhcdVTN6S0uCsFc/q1yXXz9IObxe4lzfQa8q4E9MIvahmqCiKgmqGYY13sWTH\nhMZ8dvJ/PaUS6/CVfaIpTn+uB/Frq16UmuOptVwW9dKmDaXN/aLheSPZRLEBgKaed4CF14nGclZM\n+bOxKYXjdftq8NQqJie+6Sn11du+PxJHq3Ge6EB++l7VDBVFUVDNsFKx/W5JWhR13m5eaCuhZdy2\n4Hj27/UXA/BkiymAd+U4yqfOT1fL6mn8UtUKAeqO/dXzXnZNI/KDOv4XKEzZc+uZACSML2jnS39b\nNMLmTWRv97jEH4q8z51f30TqMv96P2hnWAnIuFqSa3tzJUP+zenZBepPaz3luDpwwC1Z33o9fz8A\nDZdWjV0Q/sbZQDIPNo7MmzJHHK1KDjOl57lF5wMwMF+2wLaDxJd1XiMZ2IdcMw2A4bXWEGkkhJcE\nI4b8E1fvs5z27h0ApD7sn0WT/Og0WVEUhRBohiYigugax0quqOSyrbc4veR3K/BG6ihNcFc3bv5v\nuwR3bfLtLsB/Rueqhnd3yoDYrzwlqk8URdOx4pj+a2fR6s6Izs6b+g71nQK7KfyZ9i70jZsqWmby\nE38Cgdk/r/9JRVEUQqAZOlok8lfXt3zKDriP0niami8Lw5lQh2tPK1/Mt/y81PgXAH6aKomkRnXv\nCUDO9h0nfO+qiHeL2QH3USIOqZ5dGBE/yNa5+568HYBfni48GdTxbM6R8MLP7ejNppslxmaLZWIj\nDGREpeD3QHv3c9J7dwHQ6SwJ47P5v6nETqm6+2KLI7ttcx6vP73U9S9cdgU7Znp2Snii/j50/Sdc\nHbcNgHOqHQJgVHRUYZcrpcQ7lXvvwElEfj+/hNpVm7rfSE6Yjs1GsOD2kSXUhgEjHwCg0YtzgPTi\nK/sRnSYriqIQAs3QtWcvLTzL4t5MEdU48WlguBK5bT/dF1wPwKyOE3LLt3nyH/d+X1xlUsZKrt/o\nrdtolr3B5x4TX+/Ix9XFv+vgaY0BiDsYvBE3nBm3vBuJlD1/R1XCtWMnAM3+tZP+/+pcYv1GhMbt\nSzVDRVEU1Om6wuNavY46ko6X/hQcVZMQLbu4iDOuXbtyj6tvkAx4avIvH1vO8/0cOy02NIIofkc1\nQ0VRFFQzVJQy4TgqS/Qv7GkPQJ23/b8tTAkN2hkqShloea8EtphJtRBLovgbnSYriqIAxlrNkqso\niqKaoaIoCtoZKoqiANoZKoqiANoZKoqiANoZKoqiANoZKoqiAPD/due5dQTznPoAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 16 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for idx in range(16):\n",
    "    plt.subplot(4, 4, idx + 1)\n",
    "    plt.axis('off')\n",
    "    plt.title('[{}]'.format(mnist.train.labels[idx]))\n",
    "    plt.imshow(mnist.train.images[idx].reshape((28,28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder('float', [None, 784], name='x')\n",
    "y = tf.placeholder('float', [None, 10], name='y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_image = tf.reshape(x, [-1, 28, 28, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('conv1'):\n",
    "    C1 = tf.contrib.slim.conv2d(\n",
    "        x_image, 6, [5, 5], padding='VALID', activation_fn=tf.nn.relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('pool1'):\n",
    "    S2 = tf.contrib.slim.max_pool2d(C1, [2, 2], stride=[2, 2], padding= 'VALID')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('conv2'):\n",
    "    C3 = tf.contrib.slim.conv2d(\n",
    "        S2, 16, [5, 5], padding='VALID', activation_fn=tf.nn.relu)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('pool2'):\n",
    "    S4 = tf.contrib.slim.max_pool2d(C3, [2, 2], stride=[2, 2], padding= 'VALID')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('fc1'):\n",
    "    S4_flat = tf.contrib.slim.flatten(S4)\n",
    "    C5 = tf.contrib.slim.fully_connected(\n",
    "    S4_flat, 120, activation_fn=tf.nn.relu\n",
    "    )\n",
    "\n",
    "with tf.name_scope('fc2'):\n",
    "    F6 = tf.contrib.slim.fully_connected(\n",
    "    C5, 84, activation_fn=tf.nn.relu)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope('dropout'):\n",
    "    keep_prob = tf.placeholder(name='keep_prob', dtype=tf.float32)\n",
    "    F6_drop = tf.nn.dropout(F6, keep_prob)\n",
    "\n",
    "with tf.name_scope('fc3'):\n",
    "    logits = tf.contrib.slim.fully_connected(F6_drop, 10, activation_fn=None)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv/weights:0\n",
      "INFO:tensorflow:Summary name Conv/weights:0 is illegal; using Conv/weights_0 instead.\n",
      "Conv/biases:0\n",
      "INFO:tensorflow:Summary name Conv/biases:0 is illegal; using Conv/biases_0 instead.\n",
      "Conv_1/weights:0\n",
      "INFO:tensorflow:Summary name Conv_1/weights:0 is illegal; using Conv_1/weights_0 instead.\n",
      "Conv_1/biases:0\n",
      "INFO:tensorflow:Summary name Conv_1/biases:0 is illegal; using Conv_1/biases_0 instead.\n",
      "fully_connected/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected/weights:0 is illegal; using fully_connected/weights_0 instead.\n",
      "fully_connected/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected/biases:0 is illegal; using fully_connected/biases_0 instead.\n",
      "fully_connected_1/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_1/weights:0 is illegal; using fully_connected_1/weights_0 instead.\n",
      "fully_connected_1/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_1/biases:0 is illegal; using fully_connected_1/biases_0 instead.\n",
      "fully_connected_2/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_2/weights:0 is illegal; using fully_connected_2/weights_0 instead.\n",
      "fully_connected_2/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_2/biases:0 is illegal; using fully_connected_2/biases_0 instead.\n",
      "fully_connected_3/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_3/weights:0 is illegal; using fully_connected_3/weights_0 instead.\n",
      "fully_connected_3/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_3/biases:0 is illegal; using fully_connected_3/biases_0 instead.\n",
      "Conv_2/weights:0\n",
      "INFO:tensorflow:Summary name Conv_2/weights:0 is illegal; using Conv_2/weights_0 instead.\n",
      "Conv_2/biases:0\n",
      "INFO:tensorflow:Summary name Conv_2/biases:0 is illegal; using Conv_2/biases_0 instead.\n",
      "Conv_3/weights:0\n",
      "INFO:tensorflow:Summary name Conv_3/weights:0 is illegal; using Conv_3/weights_0 instead.\n",
      "Conv_3/biases:0\n",
      "INFO:tensorflow:Summary name Conv_3/biases:0 is illegal; using Conv_3/biases_0 instead.\n",
      "fully_connected_4/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_4/weights:0 is illegal; using fully_connected_4/weights_0 instead.\n",
      "fully_connected_4/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_4/biases:0 is illegal; using fully_connected_4/biases_0 instead.\n",
      "fully_connected_5/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_5/weights:0 is illegal; using fully_connected_5/weights_0 instead.\n",
      "fully_connected_5/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_5/biases:0 is illegal; using fully_connected_5/biases_0 instead.\n",
      "fully_connected_6/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_6/weights:0 is illegal; using fully_connected_6/weights_0 instead.\n",
      "fully_connected_6/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_6/biases:0 is illegal; using fully_connected_6/biases_0 instead.\n",
      "Conv_4/weights:0\n",
      "INFO:tensorflow:Summary name Conv_4/weights:0 is illegal; using Conv_4/weights_0 instead.\n",
      "Conv_4/biases:0\n",
      "INFO:tensorflow:Summary name Conv_4/biases:0 is illegal; using Conv_4/biases_0 instead.\n",
      "Conv_5/weights:0\n",
      "INFO:tensorflow:Summary name Conv_5/weights:0 is illegal; using Conv_5/weights_0 instead.\n",
      "Conv_5/biases:0\n",
      "INFO:tensorflow:Summary name Conv_5/biases:0 is illegal; using Conv_5/biases_0 instead.\n",
      "fully_connected_7/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_7/weights:0 is illegal; using fully_connected_7/weights_0 instead.\n",
      "fully_connected_7/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_7/biases:0 is illegal; using fully_connected_7/biases_0 instead.\n",
      "fully_connected_8/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_8/weights:0 is illegal; using fully_connected_8/weights_0 instead.\n",
      "fully_connected_8/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_8/biases:0 is illegal; using fully_connected_8/biases_0 instead.\n",
      "fully_connected_9/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_9/weights:0 is illegal; using fully_connected_9/weights_0 instead.\n",
      "fully_connected_9/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_9/biases:0 is illegal; using fully_connected_9/biases_0 instead.\n",
      "Conv_6/weights:0\n",
      "INFO:tensorflow:Summary name Conv_6/weights:0 is illegal; using Conv_6/weights_0 instead.\n",
      "Conv_6/biases:0\n",
      "INFO:tensorflow:Summary name Conv_6/biases:0 is illegal; using Conv_6/biases_0 instead.\n",
      "Conv_7/weights:0\n",
      "INFO:tensorflow:Summary name Conv_7/weights:0 is illegal; using Conv_7/weights_0 instead.\n",
      "Conv_7/biases:0\n",
      "INFO:tensorflow:Summary name Conv_7/biases:0 is illegal; using Conv_7/biases_0 instead.\n",
      "fully_connected_10/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_10/weights:0 is illegal; using fully_connected_10/weights_0 instead.\n",
      "fully_connected_10/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_10/biases:0 is illegal; using fully_connected_10/biases_0 instead.\n",
      "fully_connected_11/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_11/weights:0 is illegal; using fully_connected_11/weights_0 instead.\n",
      "fully_connected_11/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_11/biases:0 is illegal; using fully_connected_11/biases_0 instead.\n",
      "fully_connected_12/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_12/weights:0 is illegal; using fully_connected_12/weights_0 instead.\n",
      "fully_connected_12/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_12/biases:0 is illegal; using fully_connected_12/biases_0 instead.\n",
      "Conv_8/weights:0\n",
      "INFO:tensorflow:Summary name Conv_8/weights:0 is illegal; using Conv_8/weights_0 instead.\n",
      "Conv_8/biases:0\n",
      "INFO:tensorflow:Summary name Conv_8/biases:0 is illegal; using Conv_8/biases_0 instead.\n",
      "Conv_9/weights:0\n",
      "INFO:tensorflow:Summary name Conv_9/weights:0 is illegal; using Conv_9/weights_0 instead.\n",
      "Conv_9/biases:0\n",
      "INFO:tensorflow:Summary name Conv_9/biases:0 is illegal; using Conv_9/biases_0 instead.\n",
      "fully_connected_13/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_13/weights:0 is illegal; using fully_connected_13/weights_0 instead.\n",
      "fully_connected_13/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_13/biases:0 is illegal; using fully_connected_13/biases_0 instead.\n",
      "fully_connected_14/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_14/weights:0 is illegal; using fully_connected_14/weights_0 instead.\n",
      "fully_connected_14/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_14/biases:0 is illegal; using fully_connected_14/biases_0 instead.\n",
      "fully_connected_15/weights:0\n",
      "INFO:tensorflow:Summary name fully_connected_15/weights:0 is illegal; using fully_connected_15/weights_0 instead.\n",
      "fully_connected_15/biases:0\n",
      "INFO:tensorflow:Summary name fully_connected_15/biases:0 is illegal; using fully_connected_15/biases_0 instead.\n"
     ]
    }
   ],
   "source": [
    "cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "logits=logits, labels=y))\n",
    "l2_loss = tf.add_n([tf.nn.l2_loss(w)\n",
    "                  for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "                   ])\n",
    "\n",
    "cross_entropy_loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "\n",
    "l2_loss = tf.add_n([\n",
    "    tf.nn.l2_loss(w)\n",
    "    for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "])\n",
    "\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(w.name)\n",
    "    tf.summary.histogram(w.name, w)\n",
    "    \n",
    "total_loss = cross_entropy_loss + 7e-5 * l2_loss\n",
    "tf.summary.scalar('cross_entropy_loss', cross_entropy_loss)\n",
    "tf.summary.scalar('l2_loss', l2_loss)\n",
    "tf.summary.scalar('total_loss', total_loss)\n",
    "\n",
    "optimizer = tf.train.GradientDescentOptimizer(\n",
    "    learning_rate=0.3).minimize(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = tf.nn.softmax(logits)\n",
    "correct_pred = tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "trainig_step = 1100\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot feed value of shape (100,) for Tensor 'y_5:0', which has shape '(?, 10)'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-130-a8947bbafa7a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m         _,loss,rs = sess.run(\n\u001b[1;32m     17\u001b[0m         \u001b[0;34m[\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcross_entropy_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmerged\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mkeep_prob\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.6\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         )\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tf1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1098\u001b[0m                 \u001b[0;34m'Cannot feed value of shape %r for Tensor %r, '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1099\u001b[0m                 \u001b[0;34m'which has shape %r'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m                 % (np_val.shape, subfeed_t.name, str(subfeed_t.get_shape())))\n\u001b[0m\u001b[1;32m   1101\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_feedable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubfeed_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Tensor %s may not be fed.'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0msubfeed_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot feed value of shape (100,) for Tensor 'y_5:0', which has shape '(?, 10)'"
     ]
    }
   ],
   "source": [
    "merged = tf.summary.merge_all()\n",
    "with tf.Session() as sess:\n",
    "    writer = tf.summary.FileWriter('logs/', sess.graph)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    validate_data = {\n",
    "        x: mnist.validation.images,\n",
    "        y: mnist.validation.labels,\n",
    "        keep_prob: 1.0\n",
    "    }\n",
    "    \n",
    "    test_data = {x: mnist.test.images, y: mnist.test.labels, keep_prob: 1.0}\n",
    "    \n",
    "    for i in range(trainig_step):\n",
    "        xs,ys = mnist.train.next_batch(batch_size)\n",
    "        _,loss,rs = sess.run(\n",
    "        [optimizer, cross_entropy_loss, merged],\n",
    "        feed_dict={x: xs, y: ys,keep_prob: 0.6}\n",
    "        )\n",
    "        \n",
    "        writer.add_summary(rs, i)\n",
    "        \n",
    "        if i>0 and i%100==0:\n",
    "            validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "            print(\"after %d training steps, the loss is %g, the validation accuracy is %g\" % (i, loss, validate_accuracy))\n",
    "            saver.save(sess, './model.ckpt', global_step=i)\n",
    "                 \n",
    "    print('the training is finish!')\n",
    "    \n",
    "    acc = sess.run(accuracy, feed_dict=test_data)\n",
    "    print('the test accuracy is:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    ckpt = tf.train.get_checkpoint_state('./')\n",
    "    if ckpt and ckpt.model_checkpoint_path:\n",
    "        saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "        final_pred, acc = sess.run(\n",
    "            [pred, accuracy],\n",
    "            feed_dict={\n",
    "                x: mnist.test.images[:16],\n",
    "                y: mnist.test.labels[:16],\n",
    "                keep_prob: 1.0\n",
    "            })\n",
    "        orders = np.argsort(final_pred)\n",
    "        plt.figure(figsize=(8, 8))\n",
    "        print(acc)\n",
    "        for idx in range(16):\n",
    "            order = orders[idx, :][-1]\n",
    "            prob = final_pred[idx, :][order]\n",
    "            plt.subplot(4, 4, idx + 1)\n",
    "            plt.axis('off')\n",
    "            plt.title('{}: [{}]-[{:.1f}%]'.format(\n",
    "                np.argmax(mnist.test.labels[idx]), order, prob * 100))\n",
    "            plt.imshow(mnist.test.images[idx].reshape((28, 28)))\n",
    "\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.卷积的特性描述20分。\n",
    "    一方面减少了权值的数量使得网络易于优化\n",
    "    另一方面降低了模型的复杂度，也就是减小了过拟合的风险"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.为什么卷积的效果要比全连接网络好，给出至少2点理由，每点10分 \n",
    " 如果用全连接神经网络处理大尺寸图像具有三个明显的缺点：\n",
    "\n",
    "  （1）将图像展开为向量会丢失空间信息；\n",
    "\n",
    "  （2）参数过多效率低下，训练困难；\n",
    "\n",
    "  （3）大量的参数也很快会导致网络过拟合。\n",
    "\n",
    "而使用卷积神经网络可以很好地解决上面的三个问题。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python_tf 3",
   "language": "python",
   "name": "tfkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
